For the IWPR Q1 Update (Jan 2025)

Query and load IM3 data

Run this query_im3_scen("energy") only once to query from remote IM3 databases. Once a .dat file is created, we can load the existing project data by loadProject(proj = "im3scen_energy.dat").

# query the data
# im3_energy <- query_im3_scen("energy")
# load the data
im3_energy <- loadProject(proj = paste0("../", data_dir, "im3scen_energy.dat"))
# scenarios and queries 
listScenarios(im3_energy)
[1] "rcp45cooler_ssp3" "rcp45cooler_ssp5" "rcp45hotter_ssp3" "rcp45hotter_ssp5" "rcp85cooler_ssp3" "rcp85cooler_ssp5"
[7] "rcp85hotter_ssp3" "rcp85hotter_ssp5"
listQueries(im3_energy)
[1] "USA inputs by tech"                 "USA outputs by tech"                "inputs by subsector (non-electric)"
[4] "elec gen by subsector"              "USA regional natural gas outputs"   "elec energy input by subsector"    
# mappings 
source_mapping_e <- read_csv(paste0("../", data_dir, "mappings/source_mapping_e.csv"))
Rows: 88 Columns: 2-- Column specification --------------------------------------------------------------------------------------------------------
Delimiter: ","
chr (2): input, Source
i Use `spec()` to retrieve the full column specification for this data.
i Specify the column types or set `show_col_types = FALSE` to quiet this message.
target_mapping_e <- read_csv(paste0("../", data_dir, "mappings/target_mapping_e.csv"))
Rows: 104 Columns: 2-- Column specification --------------------------------------------------------------------------------------------------------
Delimiter: ","
chr (2): sector, Target
i Use `spec()` to retrieve the full column specification for this data.
i Specify the column types or set `show_col_types = FALSE` to quiet this message.
node_mapping_e <- read_csv(paste0("../", data_dir, "mappings/node_mapping_e.csv")) 
Rows: 19 Columns: 5-- Column specification --------------------------------------------------------------------------------------------------------
Delimiter: ","
chr (4): label, stage, hex, color_name
dbl (1): node
i Use `spec()` to retrieve the full column specification for this data.
i Specify the column types or set `show_col_types = FALSE` to quiet this message.

Energy Sankey

# get queries 
inputsByTechUSA <- getQuery(im3_energy, "USA inputs by tech") 
outputsByTechUSA <- getQuery(im3_energy, "USA outputs by tech")

inputBySubsectorNonElec <- getQuery(im3_energy, 'inputs by subsector (non-electric)') %>% filter_CONUSregions() 
elecEnergyInputBySubsector <- getQuery(im3_energy, 'elec energy input by subsector') %>% filter(Units == "EJ") %>% filter_CONUSregions() # in case no filtering of ELEC_RPS credits
elecGenBySubsector <- getQuery(im3_energy, 'elec gen by subsector') %>% filter(Units == "EJ") %>% filter_CONUSregions() # in case no filtering of ELEC_RPS credits
natGasOutputs <- getQuery(im3_energy, 'USA regional natural gas outputs')
datatables_energy <- list(
  "inputsByTechUSA" = inputsByTechUSA,
  "outputsByTechUSA" = outputsByTechUSA,
  "inputBySubsectorNonElec" = inputBySubsectorNonElec,
  "elecEnergyInputBySubsector" = elecEnergyInputBySubsector,
  "elecGenBySubsector" = elecGenBySubsector,
  "natGasOutputs" = natGasOutputs
)

# print column names of each datatable
lapply(datatables_energy, function(x) colnames(x))
$inputsByTechUSA
[1] "Units"      "scenario"   "region"     "sector"     "subsector"  "technology" "input"      "year"       "value"     

$outputsByTechUSA
[1] "Units"      "scenario"   "region"     "sector"     "subsector"  "technology" "output"     "year"       "value"     

$inputBySubsectorNonElec
[1] "Units"     "scenario"  "region"    "sector"    "subsector" "input"     "year"      "value"    

$elecEnergyInputBySubsector
[1] "Units"     "scenario"  "region"    "sector"    "subsector" "input"     "year"      "value"    

$elecGenBySubsector
[1] "Units"     "scenario"  "region"    "subsector" "year"      "value"    

$natGasOutputs
[1] "Units"      "scenario"   "region"     "sector"     "technology" "output"     "year"       "value"     
# print the first few rows of each datatable
lapply(datatables_energy, function(x) (x))
$inputsByTechUSA

$outputsByTechUSA

$inputBySubsectorNonElec

$elecEnergyInputBySubsector

$elecGenBySubsector

$natGasOutputs
NA

Let’s process each piece to prepare the format of: scenario, source, target, year, value. Scenario and year could be filtered for each Sankey.

Non-electricity

unique((inputBySubsectorNonElec %>% remove_month_day_night_superpeak("sector"))$sector)
  [1] "biomass liquids"           "cement"                    "coal to liquids"           "comm cooking"             
  [5] "comm cooling"              "comm heating"              "comm hot water"            "comm lighting"            
  [9] "comm non-building"         "comm office"               "comm other"                "comm refrigeration"       
 [13] "comm ventilation"          "delivered biomass"         "elect_td_ind"              "elect_td_trn"             
 [17] "gas to liquids"            "industrial energy use"     "industrial feedstocks"     "industry"                 
 [21] "oil refining"              "process heat cement"       "regional biomass"          "regional biomassOil"      
 [25] "regional corn for ethanol" "resid clothes dryers"      "resid clothes washers"     "resid computers"          
 [29] "resid cooking"             "resid cooling"             "resid dishwashers"         "resid freezers"           
 [33] "resid furnace fans"        "resid heating"             "resid hot water"           "resid lighting"           
 [37] "resid other"               "resid refrigerators"       "resid televisions"         "trn_aviation_intl"        
 [41] "trn_freight"               "trn_freight_road"          "trn_pass"                  "trn_pass_road"            
 [45] "trn_pass_road_LDV"         "trn_pass_road_LDV_4W"      "trn_shipping_intl"         "N fertilizer"             
 [49] "carbon-storage"            "municipal water"           "water_td_an_C"             "water_td_an_W"            
 [53] "water_td_dom_C"            "water_td_dom_W"            "water_td_elec_C"           "water_td_elec_W"          
 [57] "water_td_ind_C"            "water_td_ind_W"            "water_td_irr_TennR_C"      "water_td_irr_TennR_W"     
 [61] "water_td_irr_UsaCstSE_C"   "water_td_irr_UsaCstSE_W"   "water_td_pri_C"            "water_td_pri_W"           
 [65] "water_td_irr_ArkWhtRedR_C" "water_td_irr_ArkWhtRedR_W" "water_td_irr_MissppRS_C"   "water_td_irr_MissppRS_W"  
 [69] "water_td_irr_MexCstNW_C"   "water_td_irr_MexCstNW_W"   "water_td_irr_UsaColoRN_C"  "water_td_irr_UsaColoRN_W" 
 [73] "water_td_irr_UsaColoRS_C"  "water_td_irr_UsaColoRS_W"  "water_td_irr_California_C" "water_td_irr_California_W"
 [77] "water_td_irr_GreatBasin_C" "water_td_irr_GreatBasin_W" "water_td_irr_UsaPacNW_C"   "water_td_irr_UsaPacNW_W"  
 [81] "water_td_irr_MissouriR_C"  "water_td_irr_MissouriR_W"  "water_td_irr_RioGrande_C"  "water_td_irr_RioGrande_W" 
 [85] "water_td_irr_UsaCstE_C"    "water_td_irr_UsaCstE_W"    "water_td_irr_UsaCstNE_C"   "water_td_irr_UsaCstNE_W"  
 [89] "water_td_irr_Caribbean_C"  "water_td_irr_Caribbean_W"  "water_td_irr_MissppRN_C"   "water_td_irr_MissppRN_W"  
 [93] "water_td_irr_GreatLakes_C" "water_td_irr_GreatLakes_W" "water_td_irr_OhioR_C"      "water_td_irr_OhioR_W"     
 [97] "water_td_irr_TexasCst_C"   "water_td_irr_TexasCst_W"   "water_td_irr_NelsonR_C"    "water_td_irr_NelsonR_W"   
[101] "water_td_irr_FraserR_C"    "water_td_irr_FraserR_W"   
unique(inputBySubsectorNonElec$subsector)
 [1] "biomass liquids"           "cement"                    "coal to liquids"           "electricity"              
 [5] "gas"                       "biomass"                   "coal"                      "refined liquids"          
 [9] "delivered biomass"         "elect_td_ind"              "elect_td_trn"              "gas to liquids"           
[13] "hydrogen"                  "industry"                  "oil refining"              "regional biomass"         
[17] "regional biomassOil"       "regional corn for ethanol" "International Aviation"    "Domestic Ship"            
[21] "Freight Rail"              "Heavy truck"               "Light truck"               "Medium truck"             
[25] "Cycle"                     "Domestic Aviation"         "HSR"                       "Passenger Rail"           
[29] "Walk"                      "Bus"                       "2W and 3W"                 "Car"                      
[33] "Large Car and Truck"       "International Ship"        "offshore carbon-storage"   "onshore carbon-storage"   
[37] "municipal water"           "South Atlantic Gulf"       "Tennessee River"           "Arkansas White Red"       
[41] "Lower Mississippi River"   "Lower Colorado River"      "Mexico-Northwest Coast"    "Upper Colorado River"     
[45] "California River"          "Great"                     "Pacific Northwest"         "Missouri River"           
[49] "Rio Grande River"          "Mid Atlantic"              "New England"               "Caribbean"                
[53] "Upper Mississippi"         "Ohio River"                "Great Lakes"               "Texas Gulf Coast"         
[57] "Saskatchewan-Nelson"       "Fraser"                    "Pacific and Arctic Coast"  "road"                     
[61] "LDV"                       "4W"                       
unique((inputBySubsectorNonElec %>% remove_month_day_night_superpeak("input"))$input)
 [1] "elect_td_ind"                               "regional biomass"                          
 [3] "regional biomassOil"                        "regional corn for ethanol"                 
 [5] "wholesale gas"                              "process heat cement"                       
 [7] "regional coal"                              "elect_td_bld"                              
 [9] "delivered gas"                              "electricity domestic supply"               
[11] "delivered biomass"                          "delivered coal"                            
[13] "refined liquids enduse"                     "regional natural gas"                      
[15] "H2 enduse"                                  "refined liquids industrial"                
[17] "oil-credits"                                "industrial energy use"                     
[19] "industrial feedstocks"                      "industrial processes"                      
[21] "regional oil"                               "regional oilcrop"                          
[23] "regional corn"                              "renewable"                                 
[25] "elect_td_trn"                               "limestone"                                 
[27] "offshore carbon-storage"                    "onshore carbon-storage"                    
[29] "water_td_ind_C"                             "water_td_ind_W"                            
[31] "water_td_dom_C"                             "water_td_dom_W"                            
[33] "South Atlantic Gulf_water consumption"      "Tennessee River_water consumption"         
[35] "South Atlantic Gulf_water withdrawals"      "Tennessee River_water withdrawals"         
[37] "Arkansas White Red_water consumption"       "Lower Mississippi River_water consumption" 
[39] "Arkansas White Red_water withdrawals"       "Lower Mississippi River_water withdrawals" 
[41] "Lower Colorado River_water consumption"     "Mexico-Northwest Coast_water consumption"  
[43] "Upper Colorado River_water consumption"     "Lower Colorado River_water withdrawals"    
[45] "Mexico-Northwest Coast_water withdrawals"   "Upper Colorado River_water withdrawals"    
[47] "California River_water consumption"         "Great_water consumption"                   
[49] "Pacific Northwest_water consumption"        "California River_water withdrawals"        
[51] "desalination"                               "Great_water withdrawals"                   
[53] "Pacific Northwest_water withdrawals"        "Missouri River_water consumption"          
[55] "Rio Grande River_water consumption"         "Missouri River_water withdrawals"          
[57] "Rio Grande River_water withdrawals"         "Mid Atlantic_water consumption"            
[59] "New England_water consumption"              "Mid Atlantic_water withdrawals"            
[61] "New England_water withdrawals"              "Caribbean_water consumption"               
[63] "Caribbean_water withdrawals"                "Upper Mississippi_water consumption"       
[65] "Upper Mississippi_water withdrawals"        "Ohio River_water consumption"              
[67] "Ohio River_water withdrawals"               "Great Lakes_water consumption"             
[69] "Great Lakes_water withdrawals"              "Texas Gulf Coast_water consumption"        
[71] "Texas Gulf Coast_water withdrawals"         "Saskatchewan-Nelson_water consumption"     
[73] "Saskatchewan-Nelson_water withdrawals"      "Fraser_water consumption"                  
[75] "Pacific and Arctic Coast_water consumption" "Fraser_water withdrawals"                  
[77] "Pacific and Arctic Coast_water withdrawals" "trn_pass_road"                             
[79] "trn_pass_road_LDV"                          "trn_pass_road_LDV_4W"                      
[81] "trn_freight_road"                          
# map non electricity energy flows to major aggregated categories based on the mapping file 

inputs_by_subsector_nonelec <- inputBySubsectorNonElec %>% 
  filter(Units == 'EJ') %>% # only take energy flows
  filter(!input %in% c('regional corn', 'regional soybean')) %>% # remove crop inputs 
  # aggregate all monthly_day combinations to one category e.g., electricity domestic supply_Nov_day to electricity domestic supply
  remove_month_day_night_superpeak("sector") %>% remove_month_day_night_superpeak("input") %>%
  rbind(inputsByTechUSA %>% filter(str_detect(sector, "H2 ")) %>% select(-technology)) %>% # add H3 flows from the USA region level as IM3 doesn't model H2 at state level
  left_join(source_mapping_e, by = 'input') %>%
  left_join(target_mapping_e, by = 'sector')

# Note there are NAs in the output due to missing mappings or sectors that are
# not supposed to be targets and inputs that are not supposed# to be sources
# get hydrogen flows from USA region since IM3 version doesn't model H2 at state level 
inputsByTechUSA %>%# filter all inputs that have "H2 " in it
  filter(str_detect(sector, "H2")) %>%
  select(sector) %>% unique()
NA
NA
# things that were remapped as sources
unique((inputs_by_subsector_nonelec %>% filter(!is.na(Source)))$sector)
 [1] "biomass liquids"         "cement"                  "coal to liquids"         "comm cooking"           
 [5] "comm cooling"            "comm heating"            "comm hot water"          "comm lighting"          
 [9] "comm non-building"       "comm office"             "comm other"              "comm refrigeration"     
[13] "comm ventilation"        "delivered biomass"       "elect_td_ind"            "elect_td_trn"           
[17] "gas to liquids"          "industrial energy use"   "industrial feedstocks"   "industry"               
[21] "oil refining"            "process heat cement"     "regional biomass"        "resid clothes dryers"   
[25] "resid clothes washers"   "resid computers"         "resid cooking"           "resid cooling"          
[29] "resid dishwashers"       "resid freezers"          "resid furnace fans"      "resid heating"          
[33] "resid hot water"         "resid lighting"          "resid other"             "resid refrigerators"    
[37] "resid televisions"       "trn_aviation_intl"       "trn_freight"             "trn_freight_road"       
[41] "trn_pass"                "trn_pass_road"           "trn_pass_road_LDV"       "trn_pass_road_LDV_4W"   
[45] "trn_shipping_intl"       "N fertilizer"            "H2 central production"   "H2 distribution"        
[49] "H2 enduse"               "H2 forecourt production"
unique((inputs_by_subsector_nonelec %>% filter(!is.na(Source)))$subsector)
 [1] "biomass liquids"         "cement"                  "coal to liquids"         "electricity"            
 [5] "gas"                     "biomass"                 "coal"                    "refined liquids"        
 [9] "delivered biomass"       "elect_td_ind"            "elect_td_trn"            "gas to liquids"         
[13] "hydrogen"                "industry"                "oil refining"            "regional biomass"       
[17] "International Aviation"  "Domestic Ship"           "Freight Rail"            "Heavy truck"            
[21] "Light truck"             "Medium truck"            "Domestic Aviation"       "HSR"                    
[25] "Passenger Rail"          "Bus"                     "2W and 3W"               "Car"                    
[29] "Large Car and Truck"     "International Ship"      "nuclear"                 "H2 distribution"        
[33] "H2 forecourt production"
unique((inputs_by_subsector_nonelec %>% filter(!is.na(Source)))$input) # look at this
 [1] "elect_td_ind"                "regional biomass"            "regional biomassOil"         "regional corn for ethanol"  
 [5] "wholesale gas"               "process heat cement"         "regional coal"               "elect_td_bld"               
 [9] "delivered gas"               "electricity domestic supply" "delivered biomass"           "delivered coal"             
[13] "refined liquids enduse"      "regional natural gas"        "H2 enduse"                   "refined liquids industrial" 
[17] "industrial energy use"       "industrial feedstocks"       "industrial processes"        "regional oil"               
[21] "elect_td_trn"                "nuclearFuelGenIII"           "H2 central production"       "H2 distribution"            
[25] "H2 forecourt production"    
# things there were NOT mapped as sources 
unique((inputs_by_subsector_nonelec %>% filter(is.na(Source)))$sector)
[1] "industrial feedstocks" "regional biomassOil"   "trn_pass"             
unique((inputs_by_subsector_nonelec %>% filter(is.na(Source)))$subsector)
[1] "refined liquids"     "regional biomassOil" "Cycle"               "Walk"               
unique((inputs_by_subsector_nonelec %>% filter(is.na(Source)))$input) # look at this 
[1] "oil-credits"      "regional oilcrop" "renewable"       
# things that were remapped as targets
unique((inputs_by_subsector_nonelec %>% filter(!is.na(Target)))$sector) # look at this
 [1] "biomass liquids"         "cement"                  "coal to liquids"         "comm cooking"           
 [5] "comm cooling"            "comm heating"            "comm hot water"          "comm lighting"          
 [9] "comm non-building"       "comm office"             "comm other"              "comm refrigeration"     
[13] "comm ventilation"        "delivered biomass"       "elect_td_ind"            "elect_td_trn"           
[17] "gas to liquids"          "industrial energy use"   "industrial feedstocks"   "industry"               
[21] "oil refining"            "process heat cement"     "regional biomass"        "regional biomassOil"    
[25] "resid clothes dryers"    "resid clothes washers"   "resid computers"         "resid cooking"          
[29] "resid cooling"           "resid dishwashers"       "resid freezers"          "resid furnace fans"     
[33] "resid heating"           "resid hot water"         "resid lighting"          "resid other"            
[37] "resid refrigerators"     "resid televisions"       "trn_aviation_intl"       "trn_freight"            
[41] "trn_freight_road"        "trn_pass"                "trn_pass_road"           "trn_pass_road_LDV"      
[45] "trn_pass_road_LDV_4W"    "trn_shipping_intl"       "N fertilizer"            "H2 central production"  
[49] "H2 distribution"         "H2 enduse"               "H2 forecourt production"
unique((inputs_by_subsector_nonelec %>% filter(!is.na(Target)))$subsector)
 [1] "biomass liquids"         "cement"                  "coal to liquids"         "electricity"            
 [5] "gas"                     "biomass"                 "coal"                    "refined liquids"        
 [9] "delivered biomass"       "elect_td_ind"            "elect_td_trn"            "gas to liquids"         
[13] "hydrogen"                "industry"                "oil refining"            "regional biomass"       
[17] "regional biomassOil"     "International Aviation"  "Domestic Ship"           "Freight Rail"           
[21] "Heavy truck"             "Light truck"             "Medium truck"            "Cycle"                  
[25] "Domestic Aviation"       "HSR"                     "Passenger Rail"          "Walk"                   
[29] "Bus"                     "2W and 3W"               "Car"                     "Large Car and Truck"    
[33] "International Ship"      "nuclear"                 "H2 distribution"         "H2 forecourt production"
unique((inputs_by_subsector_nonelec %>% filter(!is.na(Target)))$input)
 [1] "elect_td_ind"                "regional biomass"            "regional biomassOil"         "regional corn for ethanol"  
 [5] "wholesale gas"               "process heat cement"         "regional coal"               "elect_td_bld"               
 [9] "delivered gas"               "electricity domestic supply" "delivered biomass"           "delivered coal"             
[13] "refined liquids enduse"      "regional natural gas"        "H2 enduse"                   "refined liquids industrial" 
[17] "oil-credits"                 "industrial energy use"       "industrial feedstocks"       "industrial processes"       
[21] "regional oil"                "regional oilcrop"            "renewable"                   "elect_td_trn"               
[25] "nuclearFuelGenIII"           "H2 central production"       "H2 distribution"             "H2 forecourt production"    
# things that were NOT remapped as targets
unique((inputs_by_subsector_nonelec %>% filter(is.na(Target)))$sector) # look at this
character(0)
unique((inputs_by_subsector_nonelec %>% filter(is.na(Target)))$subsector)
character(0)
unique((inputs_by_subsector_nonelec %>% filter(is.na(Target)))$input)
character(0)
# check for unmatched sources
inputs_by_subsector_nonelec_unmatched_source <- inputs_by_subsector_nonelec %>% 
  filter(is.na(Source)) %>% 
  select(scenario, sector, subsector, input, Source, Target) %>% 
  unique()

unique(inputs_by_subsector_nonelec_unmatched_source$input)
[1] "oil-credits"      "regional oilcrop" "renewable"       
unmatched_sources <- c("oil-credits", "renewable", "regional oilcrop", "process heat cement", "process heat dac")

if(! all(inputs_by_subsector_nonelec_unmatched_source$input %in% unmatched_sources )){
  unmatched <- setdiff(inputs_by_subsector_nonelec_unmatched_source$input, unmatched_sources)
  stop(paste0("Unmatched Sources in inputs by subsector nonelec. Check Source mapping file against gcam data: ", paste(unmatched, collapse = ' - ')))
}
# check for unmatched targets
inputs_by_subsector_nonelec_unmatched_target <- inputs_by_subsector_nonelec %>% 
  filter(is.na(Target)) %>% 
  select(scenario, sector, subsector, input, Source, Target) %>% 
  unique

unique(inputs_by_subsector_nonelec_unmatched_target$sector)
character(0)
unmatched_targets <- c("H2 central production", 
                       "H2 liquid truck",
                       "H2 pipeline",
                       "H2 wholesale delivery" #all intermediate hydrogen markets that are double counting - only want H2 industrial and H2 MHDV
                       )
if(! all(inputs_by_subsector_nonelec_unmatched_target$sector %in% unmatched_targets)){
  unmatched <- setdiff(inputs_by_subsector_nonelec_unmatched_target$sector, unmatched_targets)
  stop(paste0("Unmatched Sources in inputs by subsector nonelec. Check Source mapping file against gcam data: ", paste(unmatched, collapse = ' - ')))
}

Get other flows such as gas processing and backup electricity

gas_processing_flows <- inputsByTechUSA %>%
  filter(sector == "gas processing") %>%
  left_join(source_mapping_e, by = "input") %>%
  left_join(target_mapping_e, by = "sector") %>%
  group_by(scenario, Units, year, Source, Target) %>%
  summarize(value = sum(value)) %>%
  ungroup()
`summarise()` has grouped output by 'scenario', 'Units', 'year', 'Source'. You can override using the `.groups` argument.
backup <- inputsByTechUSA %>%
  filter(sector %in% c("backup_electricity", "csp_backup")) %>%
  left_join(source_mapping_e, by = "input") %>%
  left_join(target_mapping_e, by = "sector") %>%
  group_by(scenario, Units, year, Source, Target) %>%
  summarize(value = sum(value)) %>%
  ungroup()
`summarise()` has grouped output by 'scenario', 'Units', 'year', 'Source'. You can override using the `.groups` argument.

Electricity

elec_energy_by_subsector <- elecEnergyInputBySubsector %>% 
  filter(Units == 'EJ') %>%
  filter(!input %in% c('backup_electricity', 'csp_backup'),
         !subsector %in% c("nuclear", "geothermal")) %>% #don't want to double count electricity from backup, and nuclear and geothermal are reported from output
  left_join(target_mapping_e, by = 'sector') %>% 
  left_join(source_mapping_e, by = 'input') 

#hydropower is only available as an output. In the "direct equivalent" reporting convention used here, input = output
hydro_power <- elecGenBySubsector %>%
  filter(subsector == 'hydro') %>%
  mutate(Source = 'Hydropower',
         Target = 'Electricity')

# nuclear's reported thermal inputs assume a 3:1 conversion, so for "direct equivalent" reporting we use the output
nuclear <- elecGenBySubsector %>%
  filter(subsector == 'nuclear') %>%
  mutate(Source = 'Nuclear',
         Target = 'Electricity') 

# geothermal's reported thermal inputs assume a 10:1 conversion, so for "direct equivalent" reporting we use the output
geothermal <- elecGenBySubsector %>%
  filter(subsector == 'geothermal') %>%
  mutate(Source = 'Geothermal',
         Target = 'Electricity')
# put everything together
all_energy <- inputs_by_subsector_nonelec %>% 
  bind_rows(gas_processing_flows) %>% 
  bind_rows(backup) %>%
  bind_rows(elec_energy_by_subsector) %>% 
  bind_rows(hydro_power) %>% 
  bind_rows(nuclear) %>%
  bind_rows(geothermal)

Source_Target_all <- all_energy %>% 
  group_by(scenario, Units, Source, Target, year) %>%
  summarise(value = sum(value))  %>% 
  filter( Source != Target) %>% 
  filter( Target != 'Biomass') %>% 
  ungroup() 
`summarise()` has grouped output by 'scenario', 'Units', 'Source', 'Target'. You can override using the `.groups` argument.
datatable(Source_Target_all, filter = 'top', rownames = FALSE)
# calculate losses 
# take the different of the sum of sources of a node and the sum of targets of a node and assign it to Losses target node. The Source would be the node it self. 

losses <- Source_Target_all %>%
  # cather data to have one column for "direction" (Source/Target) and one for "node"
  pivot_longer(cols = c(Source, Target), 
               names_to = "direction", 
               values_to = "node") %>%
  # only calculate for mid-tier/transformation flows
  left_join(node_mapping_e %>% filter(stage == "mid") %>% select("node" = "label", stage), by = "node" ) %>%
  filter(!is.na(stage)) %>%
  group_by(scenario, year, Units, node, direction) %>%
  summarize(total_value = sum(value), .groups = "drop") %>%
  pivot_wider(names_from = direction, values_from = total_value, values_fill = 0) %>%
  mutate(losses = Target - Source) %>%
  # filter(losses > 0) %>%
  # create the "Losses" rows with losses as target and the node as Source
  mutate(Source = node, Target = "Losses", value = losses) %>%
  select(scenario, Units, Source, Target, year, value) 

# add the losses back to the original dataset
Source_Target_all %>% bind_rows(losses) -> Source_Target_all_losses
# energy losses plot 
if (F) {
  losses %>% 
  ggplot(aes(x = year, y = value, color = Source, linetype = Source)) +
  geom_line(size = 1) +
  scale_color_manual(values = node_mapping_e %>% filter(stage == "mid") %>% pull(hex)) +
  facet_wrap(~scenario, nrow = 2) +
  labs(title = "Energy Efficiency Losses: IM3 scenarios", x = "Year", y = "Losses (EJ)") +
  theme_bw() 
}

Plotting

scenario_name <- "rcp45cooler_ssp3"
plot_scenario_name <- 'RCP 4.5 Cooler SSP3'

select_year <- '2050'
gcam_data_unit <- 'EJ'

# sankey formatting
link_alpha <- .5

# source/target mapping

node_mapping_in <- node_mapping_e

# GCAM data
gcam_data <- Source_Target_all_losses %>% 
  filter(scenario == scenario_name) %>% filter( year == select_year) %>% select(-scenario)

all_links <- c(gcam_data$Source, gcam_data$Target) %>% unique()

node_mapping_e <- node_mapping_in %>% filter(label %in% all_links)

node_mapping_e$node <- 0:(nrow(node_mapping_e)-1)

# process node data
links_data <- gcam_data %>% 
  # filter(Source %in% c("Hydropower", "Solar")) %>% 
  select(Source, Target, value) %>% 
  # mutate(Target = ifelse(str_detect(Target, 'Ind'), 'Industry', Target)) %>% 
  group_by(Source, Target) %>% 
  summarize(value = sum(value)) %>% 
  ungroup() %>% 
  rename(Source_label = Source,
         Target_label = Target) %>% 
  left_join(node_mapping_e %>% select(label, node), by = c('Source_label' = 'label')) %>% 
  rename(Source_node = node) %>% 
  left_join(node_mapping_e %>% select(label, node), by = c('Target_label' = 'label')) %>% 
  rename(Target_node = node) %>% 
  left_join(node_mapping_e %>% select(label, stage, hex, color_name), by = c('Source_label' = 'label')) %>% 
  mutate(rgb = apply(FUN = paste, MARGIN = 2, X = col2rgb(hex), collapse = ',')) %>% 
  mutate(rgba = paste0('rgba(', rgb, ', ', link_alpha,')')) %>% 
  mutate(link_label = paste(Source_label, round(value, digits = 1),'EJ')) %>% 
  filter(value>0) %>% 
  arrange(Source_node)
`summarise()` has grouped output by 'Source'. You can override using the `.groups` argument.
datatable(links_data, filter = 'top', rownames = FALSE, options = list(pageLength = 10, scrollX = TRUE))
# process node percent labels

# source
source_sum <- links_data %>% 
  select(Source_label, value) %>% 
  left_join(node_mapping_e %>% select(label, stage), by = c('Source_label' = 'label')) %>% 
  rename(label=Source_label) %>% 
  filter(tolower(stage) == 'source') %>% 
  group_by(label, stage) %>% 
  summarize(node_sum = sum(value))
`summarise()` has grouped output by 'label'. You can override using the `.groups` argument.
source_total <- source_sum %>% 
  pull(node_sum) %>% sum

source_percent <- source_sum %>% 
  mutate(percent = node_sum/source_total*100) %>% 
  left_join(node_mapping_e) %>% 
  arrange(node) %>% 
  mutate(x = .01) %>% 
  mutate(csum_norm = source_total)
Joining with `by = join_by(label, stage)`
source_percent$csum <- cumsum(source_percent$node_sum)
source_percent$start <- lag(source_percent$csum)

# target
target_sum <- links_data %>% 
  select(Target_label, value) %>% 
  left_join(node_mapping_e %>% select(label, stage), by = c('Target_label' = 'label')) %>% 
  rename(label=Target_label) %>% 
  filter(stage == 'target') %>% 
  group_by(label, stage) %>% 
  summarize(node_sum = sum(value))
`summarise()` has grouped output by 'label'. You can override using the `.groups` argument.
target_total <- target_sum %>% 
  pull(node_sum) %>% sum

target_percent <- target_sum %>% 
  mutate(percent = node_sum/target_total*100) %>% 
  left_join(node_mapping_e) %>% 
  arrange(node) %>% 
  mutate(x = .95) %>% 
  mutate(csum_norm = target_total)
Joining with `by = join_by(label, stage)`
target_percent$csum <- cumsum(target_percent$node_sum)
target_percent$start <- lag(target_percent$csum)

# Intermediate Carriers Flows in
intermediate_nodes <- node_mapping_e %>% filter(stage == 'mid') %>% pull(label)
 intermediate_flows_in_total <- links_data %>%
   filter(Target_label %in% intermediate_nodes) %>% 
   group_by(Target_label) %>% 
   summarize(node_sum = sum(value))
 
intermediate_percent <- intermediate_flows_in_total %>% 
 rename(label = Target_label) %>% 
 mutate(stage = 'mid') %>% 
 mutate(percent =node_sum/source_total*100) %>% 
 left_join(node_mapping_e)
Joining with `by = join_by(label, stage)`
intermediate_total <- intermediate_percent %>% pull(node_sum) %>% sum

intermediate_flows_out_total <- links_data %>%
 filter(Source_label %in% intermediate_nodes) %>% 
 group_by(Source_label) %>% 
 summarize(value = sum(value))
  
# process node locations 

# final node info
nodes_data <- bind_rows(source_percent, intermediate_percent, target_percent) %>%
  arrange(node) %>%
  replace_na(list(start = 0)) %>% 
  mutate(mid_point = (start+csum)/2) %>% 
  mutate(y = mid_point/csum_norm) %>% 
  mutate(y = ifelse(label == 'Gas', 0.5,
                    ifelse(label == 'Liquid Fuels', 0.2,
                    ifelse(label == 'Electricity', 0.6,
                    ifelse(label == 'Hydrogen',0.9,y))))) %>% 
  mutate(x = ifelse(label == 'Gas', 0.25,
                    ifelse(label == 'Liquid Fuels', 0.4,
                           ifelse(label == 'Electricity', 0.6,
                                  ifelse(label == 'Hydrogen', 0.7,x))))) %>%
  mutate(node_label = ifelse(is.na(node_sum), label, 
                               paste0(label, ' ',round(node_sum, digits = 1) , gcam_data_unit, 
                                      ' ', round(percent, digits = 1),'%'))) 
  

# Check that Source and Targets in Links are in the node mapping

if( any(is.na(links_data$Source_node)) ) stop("Check Source number mapping - NA's")
if( any(is.na(links_data$Target_node)) ) stop("Check Target number mapping - NA's")
  
datatable(nodes_data, filter = 'top', rownames = FALSE, options = list(pageLength = 20, scrollX = TRUE))
# save files for Kendall
write_csv(Source_Target_all_losses, paste0("../", data_dir, 'allenergy_source_target.csv'))
write_csv(nodes_data, paste0("../", data_dir, 'allenergy_nodes_data.csv'))
write_csv(links_data, paste0("../", data_dir, 'allenergy_links_data.csv'))
# plot sankey
sankey_figure <- plot_ly( 
      type = "sankey",
      # arrangement = "snap",
      domain = list(x =  c(0,1),y =  c(0,1)),
      orientation = "h",
      valueformat = ".0f",
      valuesuffix = gcam_data_unit,

# Nodes  
      node = list( label = nodes_data %>% pull(node_label),
                   color = nodes_data %>% pull(hex),
                   x = nodes_data %>% pull(x),
                   y = nodes_data %>% pull(y),
                   pad = 3,
                   thickness = 15,
                   line = list(color = "black",width = 0.5)),
  
# Links
      link = list(source = links_data$Source_node,
                  target = links_data$Target_node,
                  value =  links_data$value,
                  color =  links_data$rgba)
) 

# add Formatting
plot_title <- paste0('Energy - ', plot_scenario_name, ' - ',select_year)
sankey_figure <- sankey_figure %>% layout(
  title = plot_title,
  font = list(size = 11),
  xaxis = list(showgrid = F, zeroline = F),
  yaxis = list(showgrid = F, zeroline = F))

sankey_figure
NA
NA
LS0tDQp0aXRsZTogIkVuZXJneSBGbG93cyBmcm9tIHRoZSBJTTMgR0NBTS1VU0EgU2NlbmFyaW9zIg0KYXV0aG9yOiAiSGFzc2FuIE5pYXppIChoYXNzYW4ubmlhemlAcG5ubC5nb3YpIHwgQWRhcHRlZCBmcm9tIHRoZSB3b3JrIG9mIFJhY2hlbCBIb2VzbHkiDQpkYXRlOiAiTGFzdCBjb21waWxlZCBvbiBgciBmb3JtYXQoU3lzLnRpbWUoKSwgJyVkICVCLCAlWScpYCINCm91dHB1dDoNCiAgaHRtbF9ub3RlYm9vazoNCiAgICB0b2M6IHRydWUNCiAgICAjIHRvY19mbG9hdDogVFJVRQ0KICBodG1sX2RvY3VtZW50Og0KICAgIHRvYzogdHJ1ZQ0KICAgIGRmX3ByaW50OiBwYWdlZA0KLS0tDQoNCmBgYHtyIHNldHVwLCBpbmNsdWRlPUZBTFNFLCB3YXJuaW5nPUZBTFNFfQ0KIyBieSBkZWZhdWx0IGNvbGxhcHNlL2hpZGUgdGhlIGNvZGUNCiMga25pdHI6Om9wdHNfY2h1bmskc2V0KGVjaG8gPSBGQUxTRSkNCiMgc2V0IHdvcmtpbmcgZGlyZWN0b3J5IHRvIG9uZSBmb2xkZXIgdXANCnNldHdkKCIuLi8iKQ0KIyBnZXR3ZCgpDQpzb3VyY2UoIi4vUi9mdW5jdGlvbnMuUiIpDQpgYGANCg0KIyMjIEZvciB0aGUgSVdQUiBRMSBVcGRhdGUgKEphbiAyMDI1KQ0KDQotICAgR29hbDogcGxvdCBhbiBhbGwgZW5lcmd5IHNhbmtleSBmb3IgUTEgdXBkYXRlIG9mIHRoZSBFVy1GbG93cyBwcm9qZWN0DQoNCiMjIyBRdWVyeSBhbmQgbG9hZCBJTTMgZGF0YQ0KDQpSdW4gdGhpcyBgcXVlcnlfaW0zX3NjZW4oImVuZXJneSIpYCBvbmx5IG9uY2UgdG8gcXVlcnkgZnJvbSByZW1vdGUgSU0zIGRhdGFiYXNlcy4gT25jZSBhIGAuZGF0YCBmaWxlIGlzIGNyZWF0ZWQsIHdlIGNhbiBsb2FkIHRoZSBleGlzdGluZyBwcm9qZWN0IGRhdGEgYnkgYGxvYWRQcm9qZWN0KHByb2ogPSAiaW0zc2Nlbl9lbmVyZ3kuZGF0IilgLg0KDQpgYGB7ciBtZXNzYWdlPUZBTFNFLCB3YXJuaW5nPUZBTFNFfQ0KIyBxdWVyeSB0aGUgZGF0YQ0KIyBpbTNfZW5lcmd5IDwtIHF1ZXJ5X2ltM19zY2VuKCJlbmVyZ3kiKQ0KYGBgDQoNCmBgYHtyLCB3YXJuaW5nID0gRkFMU0V9DQojIGxvYWQgdGhlIGRhdGENCmltM19lbmVyZ3kgPC0gbG9hZFByb2plY3QocHJvaiA9IHBhc3RlMCgiLi4vIiwgZGF0YV9kaXIsICJpbTNzY2VuX2VuZXJneS5kYXQiKSkNCmBgYA0KDQpgYGB7cn0NCiMgc2NlbmFyaW9zIGFuZCBxdWVyaWVzIA0KbGlzdFNjZW5hcmlvcyhpbTNfZW5lcmd5KQ0KbGlzdFF1ZXJpZXMoaW0zX2VuZXJneSkNCmBgYA0KDQoNCmBgYHtyLCB3YXJuaW5nID0gRkFMU0V9DQojIG1hcHBpbmdzIA0Kc291cmNlX21hcHBpbmdfZSA8LSByZWFkX2NzdihwYXN0ZTAoIi4uLyIsIGRhdGFfZGlyLCAibWFwcGluZ3Mvc291cmNlX21hcHBpbmdfZS5jc3YiKSkNCnRhcmdldF9tYXBwaW5nX2UgPC0gcmVhZF9jc3YocGFzdGUwKCIuLi8iLCBkYXRhX2RpciwgIm1hcHBpbmdzL3RhcmdldF9tYXBwaW5nX2UuY3N2IikpDQpub2RlX21hcHBpbmdfZSA8LSByZWFkX2NzdihwYXN0ZTAoIi4uLyIsIGRhdGFfZGlyLCAibWFwcGluZ3Mvbm9kZV9tYXBwaW5nX2UuY3N2IikpIA0KYGBgDQoNCg0KIyMjIEVuZXJneSBTYW5rZXkNCg0KYGBge3IsIHdhcm5pbmcgPSBGQUxTRX0NCiMgZ2V0IHF1ZXJpZXMgDQppbnB1dHNCeVRlY2hVU0EgPC0gZ2V0UXVlcnkoaW0zX2VuZXJneSwgIlVTQSBpbnB1dHMgYnkgdGVjaCIpIA0Kb3V0cHV0c0J5VGVjaFVTQSA8LSBnZXRRdWVyeShpbTNfZW5lcmd5LCAiVVNBIG91dHB1dHMgYnkgdGVjaCIpDQoNCmlucHV0QnlTdWJzZWN0b3JOb25FbGVjIDwtIGdldFF1ZXJ5KGltM19lbmVyZ3ksICdpbnB1dHMgYnkgc3Vic2VjdG9yIChub24tZWxlY3RyaWMpJykgJT4lIGZpbHRlcl9DT05VU3JlZ2lvbnMoKSANCmVsZWNFbmVyZ3lJbnB1dEJ5U3Vic2VjdG9yIDwtIGdldFF1ZXJ5KGltM19lbmVyZ3ksICdlbGVjIGVuZXJneSBpbnB1dCBieSBzdWJzZWN0b3InKSAlPiUgZmlsdGVyKFVuaXRzID09ICJFSiIpICU+JSBmaWx0ZXJfQ09OVVNyZWdpb25zKCkgIyBpbiBjYXNlIG5vIGZpbHRlcmluZyBvZiBFTEVDX1JQUyBjcmVkaXRzDQplbGVjR2VuQnlTdWJzZWN0b3IgPC0gZ2V0UXVlcnkoaW0zX2VuZXJneSwgJ2VsZWMgZ2VuIGJ5IHN1YnNlY3RvcicpICU+JSBmaWx0ZXIoVW5pdHMgPT0gIkVKIikgJT4lIGZpbHRlcl9DT05VU3JlZ2lvbnMoKSAjIGluIGNhc2Ugbm8gZmlsdGVyaW5nIG9mIEVMRUNfUlBTIGNyZWRpdHMNCm5hdEdhc091dHB1dHMgPC0gZ2V0UXVlcnkoaW0zX2VuZXJneSwgJ1VTQSByZWdpb25hbCBuYXR1cmFsIGdhcyBvdXRwdXRzJykNCg0KYGBgDQoNCmBgYHtyLCBtZXNzc2FnZSA9IFQsIHdhcm5pbmcgPSBGQUxTRX0NCmRhdGF0YWJsZXNfZW5lcmd5IDwtIGxpc3QoDQogICJpbnB1dHNCeVRlY2hVU0EiID0gaW5wdXRzQnlUZWNoVVNBLA0KICAib3V0cHV0c0J5VGVjaFVTQSIgPSBvdXRwdXRzQnlUZWNoVVNBLA0KICAiaW5wdXRCeVN1YnNlY3Rvck5vbkVsZWMiID0gaW5wdXRCeVN1YnNlY3Rvck5vbkVsZWMsDQogICJlbGVjRW5lcmd5SW5wdXRCeVN1YnNlY3RvciIgPSBlbGVjRW5lcmd5SW5wdXRCeVN1YnNlY3RvciwNCiAgImVsZWNHZW5CeVN1YnNlY3RvciIgPSBlbGVjR2VuQnlTdWJzZWN0b3IsDQogICJuYXRHYXNPdXRwdXRzIiA9IG5hdEdhc091dHB1dHMNCikNCg0KIyBwcmludCBjb2x1bW4gbmFtZXMgb2YgZWFjaCBkYXRhdGFibGUNCmxhcHBseShkYXRhdGFibGVzX2VuZXJneSwgZnVuY3Rpb24oeCkgY29sbmFtZXMoeCkpDQoNCiMgcHJpbnQgdGhlIGZpcnN0IGZldyByb3dzIG9mIGVhY2ggZGF0YXRhYmxlDQpsYXBwbHkoZGF0YXRhYmxlc19lbmVyZ3ksIGZ1bmN0aW9uKHgpICh4KSkNCmBgYA0KDQpMZXQncyBwcm9jZXNzIGVhY2ggcGllY2UgdG8gcHJlcGFyZSB0aGUgZm9ybWF0IG9mOiBzY2VuYXJpbywgc291cmNlLCB0YXJnZXQsIHllYXIsIHZhbHVlLiBTY2VuYXJpbyBhbmQgeWVhciBjb3VsZCBiZSBmaWx0ZXJlZCBmb3IgZWFjaCBTYW5rZXkuDQoNCiMjIyMgTm9uLWVsZWN0cmljaXR5IA0KDQoNCmBgYHtyIGZpZy53aWR0aD04LCB3YXJuaW5nPUZBTFNFfQ0KdW5pcXVlKChpbnB1dEJ5U3Vic2VjdG9yTm9uRWxlYyAlPiUgcmVtb3ZlX21vbnRoX2RheV9uaWdodF9zdXBlcnBlYWsoInNlY3RvciIpKSRzZWN0b3IpDQp1bmlxdWUoaW5wdXRCeVN1YnNlY3Rvck5vbkVsZWMkc3Vic2VjdG9yKQ0KdW5pcXVlKChpbnB1dEJ5U3Vic2VjdG9yTm9uRWxlYyAlPiUgcmVtb3ZlX21vbnRoX2RheV9uaWdodF9zdXBlcnBlYWsoImlucHV0IikpJGlucHV0KQ0KYGBgDQoNCg0KYGBge3IgZmlnLndpZHRoPTgsIHdhcm5pbmc9RkFMU0V9DQojIG1hcCBub24gZWxlY3RyaWNpdHkgZW5lcmd5IGZsb3dzIHRvIG1ham9yIGFnZ3JlZ2F0ZWQgY2F0ZWdvcmllcyBiYXNlZCBvbiB0aGUgbWFwcGluZyBmaWxlIA0KDQppbnB1dHNfYnlfc3Vic2VjdG9yX25vbmVsZWMgPC0gaW5wdXRCeVN1YnNlY3Rvck5vbkVsZWMgJT4lIA0KICBmaWx0ZXIoVW5pdHMgPT0gJ0VKJykgJT4lICMgb25seSB0YWtlIGVuZXJneSBmbG93cw0KICBmaWx0ZXIoIWlucHV0ICVpbiUgYygncmVnaW9uYWwgY29ybicsICdyZWdpb25hbCBzb3liZWFuJykpICU+JSAjIHJlbW92ZSBjcm9wIGlucHV0cyANCiAgIyBhZ2dyZWdhdGUgYWxsIG1vbnRobHlfZGF5IGNvbWJpbmF0aW9ucyB0byBvbmUgY2F0ZWdvcnkgZS5nLiwgZWxlY3RyaWNpdHkgZG9tZXN0aWMgc3VwcGx5X05vdl9kYXkgdG8gZWxlY3RyaWNpdHkgZG9tZXN0aWMgc3VwcGx5DQogIHJlbW92ZV9tb250aF9kYXlfbmlnaHRfc3VwZXJwZWFrKCJzZWN0b3IiKSAlPiUgcmVtb3ZlX21vbnRoX2RheV9uaWdodF9zdXBlcnBlYWsoImlucHV0IikgJT4lDQogIHJiaW5kKGlucHV0c0J5VGVjaFVTQSAlPiUgZmlsdGVyKHN0cl9kZXRlY3Qoc2VjdG9yLCAiSDIgIikpICU+JSBzZWxlY3QoLXRlY2hub2xvZ3kpKSAlPiUgIyBhZGQgSDMgZmxvd3MgZnJvbSB0aGUgVVNBIHJlZ2lvbiBsZXZlbCBhcyBJTTMgZG9lc24ndCBtb2RlbCBIMiBhdCBzdGF0ZSBsZXZlbA0KICBsZWZ0X2pvaW4oc291cmNlX21hcHBpbmdfZSwgYnkgPSAnaW5wdXQnKSAlPiUNCiAgbGVmdF9qb2luKHRhcmdldF9tYXBwaW5nX2UsIGJ5ID0gJ3NlY3RvcicpDQoNCiMgTm90ZSB0aGVyZSBhcmUgTkFzIGluIHRoZSBvdXRwdXQgZHVlIHRvIG1pc3NpbmcgbWFwcGluZ3Mgb3Igc2VjdG9ycyB0aGF0IGFyZQ0KIyBub3Qgc3VwcG9zZWQgdG8gYmUgdGFyZ2V0cyBhbmQgaW5wdXRzIHRoYXQgYXJlIG5vdCBzdXBwb3NlZCMgdG8gYmUgc291cmNlcw0KYGBgDQoNCg0KYGBge3IgZmlnLndpZHRoPTgsIHdhcm5pbmc9RkFMU0V9DQojIGdldCBoeWRyb2dlbiBmbG93cyBmcm9tIFVTQSByZWdpb24gc2luY2UgSU0zIHZlcnNpb24gZG9lc24ndCBtb2RlbCBIMiBhdCBzdGF0ZSBsZXZlbCANCmlucHV0c0J5VGVjaFVTQSAlPiUjIGZpbHRlciBhbGwgaW5wdXRzIHRoYXQgaGF2ZSAiSDIgIiBpbiBpdA0KICBmaWx0ZXIoc3RyX2RldGVjdChzZWN0b3IsICJIMiIpKSAlPiUNCiAgc2VsZWN0KHNlY3RvcikgJT4lIHVuaXF1ZSgpDQoNCg0KYGBgDQoNCg0KYGBge3IgZmlnLndpZHRoPTgsIHdhcm5pbmc9RkFMU0V9DQojIHRoaW5ncyB0aGF0IHdlcmUgcmVtYXBwZWQgYXMgc291cmNlcw0KdW5pcXVlKChpbnB1dHNfYnlfc3Vic2VjdG9yX25vbmVsZWMgJT4lIGZpbHRlcighaXMubmEoU291cmNlKSkpJHNlY3RvcikNCnVuaXF1ZSgoaW5wdXRzX2J5X3N1YnNlY3Rvcl9ub25lbGVjICU+JSBmaWx0ZXIoIWlzLm5hKFNvdXJjZSkpKSRzdWJzZWN0b3IpDQp1bmlxdWUoKGlucHV0c19ieV9zdWJzZWN0b3Jfbm9uZWxlYyAlPiUgZmlsdGVyKCFpcy5uYShTb3VyY2UpKSkkaW5wdXQpICMgbG9vayBhdCB0aGlzDQpgYGANCg0KYGBge3IgZmlnLndpZHRoPTgsIHdhcm5pbmc9RkFMU0V9DQojIHRoaW5ncyB0aGVyZSB3ZXJlIE5PVCBtYXBwZWQgYXMgc291cmNlcyANCnVuaXF1ZSgoaW5wdXRzX2J5X3N1YnNlY3Rvcl9ub25lbGVjICU+JSBmaWx0ZXIoaXMubmEoU291cmNlKSkpJHNlY3RvcikNCnVuaXF1ZSgoaW5wdXRzX2J5X3N1YnNlY3Rvcl9ub25lbGVjICU+JSBmaWx0ZXIoaXMubmEoU291cmNlKSkpJHN1YnNlY3RvcikNCnVuaXF1ZSgoaW5wdXRzX2J5X3N1YnNlY3Rvcl9ub25lbGVjICU+JSBmaWx0ZXIoaXMubmEoU291cmNlKSkpJGlucHV0KSAjIGxvb2sgYXQgdGhpcyANCmBgYA0KDQoNCmBgYHtyIGZpZy53aWR0aD04LCB3YXJuaW5nPUZBTFNFfQ0KIyB0aGluZ3MgdGhhdCB3ZXJlIHJlbWFwcGVkIGFzIHRhcmdldHMNCnVuaXF1ZSgoaW5wdXRzX2J5X3N1YnNlY3Rvcl9ub25lbGVjICU+JSBmaWx0ZXIoIWlzLm5hKFRhcmdldCkpKSRzZWN0b3IpICMgbG9vayBhdCB0aGlzDQp1bmlxdWUoKGlucHV0c19ieV9zdWJzZWN0b3Jfbm9uZWxlYyAlPiUgZmlsdGVyKCFpcy5uYShUYXJnZXQpKSkkc3Vic2VjdG9yKQ0KdW5pcXVlKChpbnB1dHNfYnlfc3Vic2VjdG9yX25vbmVsZWMgJT4lIGZpbHRlcighaXMubmEoVGFyZ2V0KSkpJGlucHV0KQ0KYGBgDQoNCg0KYGBge3IgZmlnLndpZHRoPTgsIHdhcm5pbmc9RkFMU0V9DQojIHRoaW5ncyB0aGF0IHdlcmUgTk9UIHJlbWFwcGVkIGFzIHRhcmdldHMNCnVuaXF1ZSgoaW5wdXRzX2J5X3N1YnNlY3Rvcl9ub25lbGVjICU+JSBmaWx0ZXIoaXMubmEoVGFyZ2V0KSkpJHNlY3RvcikgIyBsb29rIGF0IHRoaXMNCnVuaXF1ZSgoaW5wdXRzX2J5X3N1YnNlY3Rvcl9ub25lbGVjICU+JSBmaWx0ZXIoaXMubmEoVGFyZ2V0KSkpJHN1YnNlY3RvcikNCnVuaXF1ZSgoaW5wdXRzX2J5X3N1YnNlY3Rvcl9ub25lbGVjICU+JSBmaWx0ZXIoaXMubmEoVGFyZ2V0KSkpJGlucHV0KQ0KYGBgDQoNCg0KYGBge3IgZmlnLndpZHRoPTgsIHdhcm5pbmc9RkFMU0V9DQojIGNoZWNrIGZvciB1bm1hdGNoZWQgc291cmNlcw0KaW5wdXRzX2J5X3N1YnNlY3Rvcl9ub25lbGVjX3VubWF0Y2hlZF9zb3VyY2UgPC0gaW5wdXRzX2J5X3N1YnNlY3Rvcl9ub25lbGVjICU+JSANCiAgZmlsdGVyKGlzLm5hKFNvdXJjZSkpICU+JSANCiAgc2VsZWN0KHNjZW5hcmlvLCBzZWN0b3IsIHN1YnNlY3RvciwgaW5wdXQsIFNvdXJjZSwgVGFyZ2V0KSAlPiUgDQogIHVuaXF1ZSgpDQoNCnVuaXF1ZShpbnB1dHNfYnlfc3Vic2VjdG9yX25vbmVsZWNfdW5tYXRjaGVkX3NvdXJjZSRpbnB1dCkNCg0KdW5tYXRjaGVkX3NvdXJjZXMgPC0gYygib2lsLWNyZWRpdHMiLCAicmVuZXdhYmxlIiwgInJlZ2lvbmFsIG9pbGNyb3AiLCAicHJvY2VzcyBoZWF0IGNlbWVudCIsICJwcm9jZXNzIGhlYXQgZGFjIikNCg0KaWYoISBhbGwoaW5wdXRzX2J5X3N1YnNlY3Rvcl9ub25lbGVjX3VubWF0Y2hlZF9zb3VyY2UkaW5wdXQgJWluJSB1bm1hdGNoZWRfc291cmNlcyApKXsNCiAgdW5tYXRjaGVkIDwtIHNldGRpZmYoaW5wdXRzX2J5X3N1YnNlY3Rvcl9ub25lbGVjX3VubWF0Y2hlZF9zb3VyY2UkaW5wdXQsIHVubWF0Y2hlZF9zb3VyY2VzKQ0KICBzdG9wKHBhc3RlMCgiVW5tYXRjaGVkIFNvdXJjZXMgaW4gaW5wdXRzIGJ5IHN1YnNlY3RvciBub25lbGVjLiBDaGVjayBTb3VyY2UgbWFwcGluZyBmaWxlIGFnYWluc3QgZ2NhbSBkYXRhOiAiLCBwYXN0ZSh1bm1hdGNoZWQsIGNvbGxhcHNlID0gJyAtICcpKSkNCn0NCmBgYA0KDQoNCmBgYHtyIGZpZy53aWR0aD04LCB3YXJuaW5nPUZBTFNFfQ0KIyBjaGVjayBmb3IgdW5tYXRjaGVkIHRhcmdldHMNCmlucHV0c19ieV9zdWJzZWN0b3Jfbm9uZWxlY191bm1hdGNoZWRfdGFyZ2V0IDwtIGlucHV0c19ieV9zdWJzZWN0b3Jfbm9uZWxlYyAlPiUgDQogIGZpbHRlcihpcy5uYShUYXJnZXQpKSAlPiUgDQogIHNlbGVjdChzY2VuYXJpbywgc2VjdG9yLCBzdWJzZWN0b3IsIGlucHV0LCBTb3VyY2UsIFRhcmdldCkgJT4lIA0KICB1bmlxdWUNCg0KdW5pcXVlKGlucHV0c19ieV9zdWJzZWN0b3Jfbm9uZWxlY191bm1hdGNoZWRfdGFyZ2V0JHNlY3RvcikNCg0KdW5tYXRjaGVkX3RhcmdldHMgPC0gYygiSDIgY2VudHJhbCBwcm9kdWN0aW9uIiwgDQogICAgICAgICAgICAgICAgICAgICAgICJIMiBsaXF1aWQgdHJ1Y2siLA0KICAgICAgICAgICAgICAgICAgICAgICAiSDIgcGlwZWxpbmUiLA0KICAgICAgICAgICAgICAgICAgICAgICAiSDIgd2hvbGVzYWxlIGRlbGl2ZXJ5IiAjYWxsIGludGVybWVkaWF0ZSBoeWRyb2dlbiBtYXJrZXRzIHRoYXQgYXJlIGRvdWJsZSBjb3VudGluZyAtIG9ubHkgd2FudCBIMiBpbmR1c3RyaWFsIGFuZCBIMiBNSERWDQogICAgICAgICAgICAgICAgICAgICAgICkNCmlmKCEgYWxsKGlucHV0c19ieV9zdWJzZWN0b3Jfbm9uZWxlY191bm1hdGNoZWRfdGFyZ2V0JHNlY3RvciAlaW4lIHVubWF0Y2hlZF90YXJnZXRzKSl7DQogIHVubWF0Y2hlZCA8LSBzZXRkaWZmKGlucHV0c19ieV9zdWJzZWN0b3Jfbm9uZWxlY191bm1hdGNoZWRfdGFyZ2V0JHNlY3RvciwgdW5tYXRjaGVkX3RhcmdldHMpDQogIHN0b3AocGFzdGUwKCJVbm1hdGNoZWQgU291cmNlcyBpbiBpbnB1dHMgYnkgc3Vic2VjdG9yIG5vbmVsZWMuIENoZWNrIFNvdXJjZSBtYXBwaW5nIGZpbGUgYWdhaW5zdCBnY2FtIGRhdGE6ICIsIHBhc3RlKHVubWF0Y2hlZCwgY29sbGFwc2UgPSAnIC0gJykpKQ0KfQ0KDQoNCmBgYA0KDQpHZXQgb3RoZXIgZmxvd3Mgc3VjaCBhcyBnYXMgcHJvY2Vzc2luZyBhbmQgYmFja3VwIGVsZWN0cmljaXR5DQoNCmBgYHtyIGZpZy53aWR0aD04LCB3YXJuaW5nPUZBTFNFfQ0KZ2FzX3Byb2Nlc3NpbmdfZmxvd3MgPC0gaW5wdXRzQnlUZWNoVVNBICU+JQ0KICBmaWx0ZXIoc2VjdG9yID09ICJnYXMgcHJvY2Vzc2luZyIpICU+JQ0KICBsZWZ0X2pvaW4oc291cmNlX21hcHBpbmdfZSwgYnkgPSAiaW5wdXQiKSAlPiUNCiAgbGVmdF9qb2luKHRhcmdldF9tYXBwaW5nX2UsIGJ5ID0gInNlY3RvciIpICU+JQ0KICBncm91cF9ieShzY2VuYXJpbywgVW5pdHMsIHllYXIsIFNvdXJjZSwgVGFyZ2V0KSAlPiUNCiAgc3VtbWFyaXplKHZhbHVlID0gc3VtKHZhbHVlKSkgJT4lDQogIHVuZ3JvdXAoKQ0KDQpiYWNrdXAgPC0gaW5wdXRzQnlUZWNoVVNBICU+JQ0KICBmaWx0ZXIoc2VjdG9yICVpbiUgYygiYmFja3VwX2VsZWN0cmljaXR5IiwgImNzcF9iYWNrdXAiKSkgJT4lDQogIGxlZnRfam9pbihzb3VyY2VfbWFwcGluZ19lLCBieSA9ICJpbnB1dCIpICU+JQ0KICBsZWZ0X2pvaW4odGFyZ2V0X21hcHBpbmdfZSwgYnkgPSAic2VjdG9yIikgJT4lDQogIGdyb3VwX2J5KHNjZW5hcmlvLCBVbml0cywgeWVhciwgU291cmNlLCBUYXJnZXQpICU+JQ0KICBzdW1tYXJpemUodmFsdWUgPSBzdW0odmFsdWUpKSAlPiUNCiAgdW5ncm91cCgpDQpgYGANCg0KIyMjIyBFbGVjdHJpY2l0eQ0KDQpgYGB7ciBmaWcud2lkdGg9OCwgd2FybmluZz1GQUxTRX0NCmVsZWNfZW5lcmd5X2J5X3N1YnNlY3RvciA8LSBlbGVjRW5lcmd5SW5wdXRCeVN1YnNlY3RvciAlPiUgDQogIGZpbHRlcihVbml0cyA9PSAnRUonKSAlPiUNCiAgZmlsdGVyKCFpbnB1dCAlaW4lIGMoJ2JhY2t1cF9lbGVjdHJpY2l0eScsICdjc3BfYmFja3VwJyksDQogICAgICAgICAhc3Vic2VjdG9yICVpbiUgYygibnVjbGVhciIsICJnZW90aGVybWFsIikpICU+JSAjZG9uJ3Qgd2FudCB0byBkb3VibGUgY291bnQgZWxlY3RyaWNpdHkgZnJvbSBiYWNrdXAsIGFuZCBudWNsZWFyIGFuZCBnZW90aGVybWFsIGFyZSByZXBvcnRlZCBmcm9tIG91dHB1dA0KICBsZWZ0X2pvaW4odGFyZ2V0X21hcHBpbmdfZSwgYnkgPSAnc2VjdG9yJykgJT4lIA0KICBsZWZ0X2pvaW4oc291cmNlX21hcHBpbmdfZSwgYnkgPSAnaW5wdXQnKSANCg0KI2h5ZHJvcG93ZXIgaXMgb25seSBhdmFpbGFibGUgYXMgYW4gb3V0cHV0LiBJbiB0aGUgImRpcmVjdCBlcXVpdmFsZW50IiByZXBvcnRpbmcgY29udmVudGlvbiB1c2VkIGhlcmUsIGlucHV0ID0gb3V0cHV0DQpoeWRyb19wb3dlciA8LSBlbGVjR2VuQnlTdWJzZWN0b3IgJT4lDQogIGZpbHRlcihzdWJzZWN0b3IgPT0gJ2h5ZHJvJykgJT4lDQogIG11dGF0ZShTb3VyY2UgPSAnSHlkcm9wb3dlcicsDQogICAgICAgICBUYXJnZXQgPSAnRWxlY3RyaWNpdHknKQ0KDQojIG51Y2xlYXIncyByZXBvcnRlZCB0aGVybWFsIGlucHV0cyBhc3N1bWUgYSAzOjEgY29udmVyc2lvbiwgc28gZm9yICJkaXJlY3QgZXF1aXZhbGVudCIgcmVwb3J0aW5nIHdlIHVzZSB0aGUgb3V0cHV0DQpudWNsZWFyIDwtIGVsZWNHZW5CeVN1YnNlY3RvciAlPiUNCiAgZmlsdGVyKHN1YnNlY3RvciA9PSAnbnVjbGVhcicpICU+JQ0KICBtdXRhdGUoU291cmNlID0gJ051Y2xlYXInLA0KICAgICAgICAgVGFyZ2V0ID0gJ0VsZWN0cmljaXR5JykgDQoNCiMgZ2VvdGhlcm1hbCdzIHJlcG9ydGVkIHRoZXJtYWwgaW5wdXRzIGFzc3VtZSBhIDEwOjEgY29udmVyc2lvbiwgc28gZm9yICJkaXJlY3QgZXF1aXZhbGVudCIgcmVwb3J0aW5nIHdlIHVzZSB0aGUgb3V0cHV0DQpnZW90aGVybWFsIDwtIGVsZWNHZW5CeVN1YnNlY3RvciAlPiUNCiAgZmlsdGVyKHN1YnNlY3RvciA9PSAnZ2VvdGhlcm1hbCcpICU+JQ0KICBtdXRhdGUoU291cmNlID0gJ0dlb3RoZXJtYWwnLA0KICAgICAgICAgVGFyZ2V0ID0gJ0VsZWN0cmljaXR5JykNCmBgYA0KDQoNCmBgYHtyIGZpZy53aWR0aD04LCB3YXJuaW5nPUZBTFNFfQ0KIyBwdXQgZXZlcnl0aGluZyB0b2dldGhlcg0KYWxsX2VuZXJneSA8LSBpbnB1dHNfYnlfc3Vic2VjdG9yX25vbmVsZWMgJT4lIA0KICBiaW5kX3Jvd3MoZ2FzX3Byb2Nlc3NpbmdfZmxvd3MpICU+JSANCiAgYmluZF9yb3dzKGJhY2t1cCkgJT4lDQogIGJpbmRfcm93cyhlbGVjX2VuZXJneV9ieV9zdWJzZWN0b3IpICU+JSANCiAgYmluZF9yb3dzKGh5ZHJvX3Bvd2VyKSAlPiUgDQogIGJpbmRfcm93cyhudWNsZWFyKSAlPiUNCiAgYmluZF9yb3dzKGdlb3RoZXJtYWwpDQoNClNvdXJjZV9UYXJnZXRfYWxsIDwtIGFsbF9lbmVyZ3kgJT4lIA0KICBncm91cF9ieShzY2VuYXJpbywgVW5pdHMsIFNvdXJjZSwgVGFyZ2V0LCB5ZWFyKSAlPiUNCiAgc3VtbWFyaXNlKHZhbHVlID0gc3VtKHZhbHVlKSkgICU+JSANCiAgZmlsdGVyKCBTb3VyY2UgIT0gVGFyZ2V0KSAlPiUgDQogIGZpbHRlciggVGFyZ2V0ICE9ICdCaW9tYXNzJykgJT4lIA0KICB1bmdyb3VwKCkgDQoNCmRhdGF0YWJsZShTb3VyY2VfVGFyZ2V0X2FsbCwgZmlsdGVyID0gJ3RvcCcsIHJvd25hbWVzID0gRkFMU0UpDQpgYGANCg0KDQpgYGB7ciBmaWcud2lkdGg9OCwgd2FybmluZz1GQUxTRX0NCiMgY2FsY3VsYXRlIGxvc3NlcyANCiMgdGFrZSB0aGUgZGlmZmVyZW50IG9mIHRoZSBzdW0gb2Ygc291cmNlcyBvZiBhIG5vZGUgYW5kIHRoZSBzdW0gb2YgdGFyZ2V0cyBvZiBhIG5vZGUgYW5kIGFzc2lnbiBpdCB0byBMb3NzZXMgdGFyZ2V0IG5vZGUuIFRoZSBTb3VyY2Ugd291bGQgYmUgdGhlIG5vZGUgaXQgc2VsZi4gDQoNCmxvc3NlcyA8LSBTb3VyY2VfVGFyZ2V0X2FsbCAlPiUNCiAgIyBjYXRoZXIgZGF0YSB0byBoYXZlIG9uZSBjb2x1bW4gZm9yICJkaXJlY3Rpb24iIChTb3VyY2UvVGFyZ2V0KSBhbmQgb25lIGZvciAibm9kZSINCiAgcGl2b3RfbG9uZ2VyKGNvbHMgPSBjKFNvdXJjZSwgVGFyZ2V0KSwgDQogICAgICAgICAgICAgICBuYW1lc190byA9ICJkaXJlY3Rpb24iLCANCiAgICAgICAgICAgICAgIHZhbHVlc190byA9ICJub2RlIikgJT4lDQogICMgb25seSBjYWxjdWxhdGUgZm9yIG1pZC10aWVyL3RyYW5zZm9ybWF0aW9uIGZsb3dzDQogIGxlZnRfam9pbihub2RlX21hcHBpbmdfZSAlPiUgZmlsdGVyKHN0YWdlID09ICJtaWQiKSAlPiUgc2VsZWN0KCJub2RlIiA9ICJsYWJlbCIsIHN0YWdlKSwgYnkgPSAibm9kZSIgKSAlPiUNCiAgZmlsdGVyKCFpcy5uYShzdGFnZSkpICU+JQ0KICBncm91cF9ieShzY2VuYXJpbywgeWVhciwgVW5pdHMsIG5vZGUsIGRpcmVjdGlvbikgJT4lDQogIHN1bW1hcml6ZSh0b3RhbF92YWx1ZSA9IHN1bSh2YWx1ZSksIC5ncm91cHMgPSAiZHJvcCIpICU+JQ0KICBwaXZvdF93aWRlcihuYW1lc19mcm9tID0gZGlyZWN0aW9uLCB2YWx1ZXNfZnJvbSA9IHRvdGFsX3ZhbHVlLCB2YWx1ZXNfZmlsbCA9IDApICU+JQ0KICBtdXRhdGUobG9zc2VzID0gVGFyZ2V0IC0gU291cmNlKSAlPiUNCiAgIyBmaWx0ZXIobG9zc2VzID4gMCkgJT4lDQogICMgY3JlYXRlIHRoZSAiTG9zc2VzIiByb3dzIHdpdGggbG9zc2VzIGFzIHRhcmdldCBhbmQgdGhlIG5vZGUgYXMgU291cmNlDQogIG11dGF0ZShTb3VyY2UgPSBub2RlLCBUYXJnZXQgPSAiTG9zc2VzIiwgdmFsdWUgPSBsb3NzZXMpICU+JQ0KICBzZWxlY3Qoc2NlbmFyaW8sIFVuaXRzLCBTb3VyY2UsIFRhcmdldCwgeWVhciwgdmFsdWUpIA0KDQojIGFkZCB0aGUgbG9zc2VzIGJhY2sgdG8gdGhlIG9yaWdpbmFsIGRhdGFzZXQNClNvdXJjZV9UYXJnZXRfYWxsICU+JSBiaW5kX3Jvd3MobG9zc2VzKSAtPiBTb3VyY2VfVGFyZ2V0X2FsbF9sb3NzZXMNCmBgYA0KDQoNCmBgYHtyIGZpZy53aWR0aD04LCB3YXJuaW5nPUZBTFNFfQ0KIyBlbmVyZ3kgbG9zc2VzIHBsb3QgDQppZiAoRikgew0KICBsb3NzZXMgJT4lIA0KICBnZ3Bsb3QoYWVzKHggPSB5ZWFyLCB5ID0gdmFsdWUsIGNvbG9yID0gU291cmNlLCBsaW5ldHlwZSA9IFNvdXJjZSkpICsNCiAgZ2VvbV9saW5lKHNpemUgPSAxKSArDQogIHNjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXMgPSBub2RlX21hcHBpbmdfZSAlPiUgZmlsdGVyKHN0YWdlID09ICJtaWQiKSAlPiUgcHVsbChoZXgpKSArDQogIGZhY2V0X3dyYXAofnNjZW5hcmlvLCBucm93ID0gMikgKw0KICBsYWJzKHRpdGxlID0gIkVuZXJneSBFZmZpY2llbmN5IExvc3NlczogSU0zIHNjZW5hcmlvcyIsIHggPSAiWWVhciIsIHkgPSAiTG9zc2VzIChFSikiKSArDQogIHRoZW1lX2J3KCkgDQp9DQoNCmBgYA0KDQojIyMjIFBsb3R0aW5nIA0KDQpgYGB7ciBmaWcud2lkdGg9OCwgd2FybmluZz1GQUxTRX0NCnNjZW5hcmlvX25hbWUgPC0gInJjcDQ1Y29vbGVyX3NzcDMiDQpwbG90X3NjZW5hcmlvX25hbWUgPC0gJ1JDUCA0LjUgQ29vbGVyIFNTUDMnDQoNCnNlbGVjdF95ZWFyIDwtICcyMDUwJw0KZ2NhbV9kYXRhX3VuaXQgPC0gJ0VKJw0KDQojIHNhbmtleSBmb3JtYXR0aW5nDQpsaW5rX2FscGhhIDwtIC41DQoNCiMgc291cmNlL3RhcmdldCBtYXBwaW5nDQoNCm5vZGVfbWFwcGluZ19pbiA8LSBub2RlX21hcHBpbmdfZQ0KDQojIEdDQU0gZGF0YQ0KZ2NhbV9kYXRhIDwtIFNvdXJjZV9UYXJnZXRfYWxsX2xvc3NlcyAlPiUgDQogIGZpbHRlcihzY2VuYXJpbyA9PSBzY2VuYXJpb19uYW1lKSAlPiUgZmlsdGVyKCB5ZWFyID09IHNlbGVjdF95ZWFyKSAlPiUgc2VsZWN0KC1zY2VuYXJpbykNCg0KYWxsX2xpbmtzIDwtIGMoZ2NhbV9kYXRhJFNvdXJjZSwgZ2NhbV9kYXRhJFRhcmdldCkgJT4lIHVuaXF1ZSgpDQoNCm5vZGVfbWFwcGluZ19lIDwtIG5vZGVfbWFwcGluZ19pbiAlPiUgZmlsdGVyKGxhYmVsICVpbiUgYWxsX2xpbmtzKQ0KDQpub2RlX21hcHBpbmdfZSRub2RlIDwtIDA6KG5yb3cobm9kZV9tYXBwaW5nX2UpLTEpDQoNCiMgcHJvY2VzcyBub2RlIGRhdGENCmxpbmtzX2RhdGEgPC0gZ2NhbV9kYXRhICU+JSANCiAgIyBmaWx0ZXIoU291cmNlICVpbiUgYygiSHlkcm9wb3dlciIsICJTb2xhciIpKSAlPiUgDQogIHNlbGVjdChTb3VyY2UsIFRhcmdldCwgdmFsdWUpICU+JSANCiAgIyBtdXRhdGUoVGFyZ2V0ID0gaWZlbHNlKHN0cl9kZXRlY3QoVGFyZ2V0LCAnSW5kJyksICdJbmR1c3RyeScsIFRhcmdldCkpICU+JSANCiAgZ3JvdXBfYnkoU291cmNlLCBUYXJnZXQpICU+JSANCiAgc3VtbWFyaXplKHZhbHVlID0gc3VtKHZhbHVlKSkgJT4lIA0KICB1bmdyb3VwKCkgJT4lIA0KICByZW5hbWUoU291cmNlX2xhYmVsID0gU291cmNlLA0KICAgICAgICAgVGFyZ2V0X2xhYmVsID0gVGFyZ2V0KSAlPiUgDQogIGxlZnRfam9pbihub2RlX21hcHBpbmdfZSAlPiUgc2VsZWN0KGxhYmVsLCBub2RlKSwgYnkgPSBjKCdTb3VyY2VfbGFiZWwnID0gJ2xhYmVsJykpICU+JSANCiAgcmVuYW1lKFNvdXJjZV9ub2RlID0gbm9kZSkgJT4lIA0KICBsZWZ0X2pvaW4obm9kZV9tYXBwaW5nX2UgJT4lIHNlbGVjdChsYWJlbCwgbm9kZSksIGJ5ID0gYygnVGFyZ2V0X2xhYmVsJyA9ICdsYWJlbCcpKSAlPiUgDQogIHJlbmFtZShUYXJnZXRfbm9kZSA9IG5vZGUpICU+JSANCiAgbGVmdF9qb2luKG5vZGVfbWFwcGluZ19lICU+JSBzZWxlY3QobGFiZWwsIHN0YWdlLCBoZXgsIGNvbG9yX25hbWUpLCBieSA9IGMoJ1NvdXJjZV9sYWJlbCcgPSAnbGFiZWwnKSkgJT4lIA0KICBtdXRhdGUocmdiID0gYXBwbHkoRlVOID0gcGFzdGUsIE1BUkdJTiA9IDIsIFggPSBjb2wycmdiKGhleCksIGNvbGxhcHNlID0gJywnKSkgJT4lIA0KICBtdXRhdGUocmdiYSA9IHBhc3RlMCgncmdiYSgnLCByZ2IsICcsICcsIGxpbmtfYWxwaGEsJyknKSkgJT4lIA0KICBtdXRhdGUobGlua19sYWJlbCA9IHBhc3RlKFNvdXJjZV9sYWJlbCwgcm91bmQodmFsdWUsIGRpZ2l0cyA9IDEpLCdFSicpKSAlPiUgDQogIGZpbHRlcih2YWx1ZT4wKSAlPiUgDQogIGFycmFuZ2UoU291cmNlX25vZGUpDQoNCmRhdGF0YWJsZShsaW5rc19kYXRhLCBmaWx0ZXIgPSAndG9wJywgcm93bmFtZXMgPSBGQUxTRSwgb3B0aW9ucyA9IGxpc3QocGFnZUxlbmd0aCA9IDEwLCBzY3JvbGxYID0gVFJVRSkpDQpgYGANCg0KDQpgYGB7ciBmaWcud2lkdGg9OCwgd2FybmluZz1GQUxTRX0NCiMgcHJvY2VzcyBub2RlIHBlcmNlbnQgbGFiZWxzDQoNCiMgc291cmNlDQpzb3VyY2Vfc3VtIDwtIGxpbmtzX2RhdGEgJT4lIA0KICBzZWxlY3QoU291cmNlX2xhYmVsLCB2YWx1ZSkgJT4lIA0KICBsZWZ0X2pvaW4obm9kZV9tYXBwaW5nX2UgJT4lIHNlbGVjdChsYWJlbCwgc3RhZ2UpLCBieSA9IGMoJ1NvdXJjZV9sYWJlbCcgPSAnbGFiZWwnKSkgJT4lIA0KICByZW5hbWUobGFiZWw9U291cmNlX2xhYmVsKSAlPiUgDQogIGZpbHRlcih0b2xvd2VyKHN0YWdlKSA9PSAnc291cmNlJykgJT4lIA0KICBncm91cF9ieShsYWJlbCwgc3RhZ2UpICU+JSANCiAgc3VtbWFyaXplKG5vZGVfc3VtID0gc3VtKHZhbHVlKSkNCg0Kc291cmNlX3RvdGFsIDwtIHNvdXJjZV9zdW0gJT4lIA0KICBwdWxsKG5vZGVfc3VtKSAlPiUgc3VtDQoNCnNvdXJjZV9wZXJjZW50IDwtIHNvdXJjZV9zdW0gJT4lIA0KICBtdXRhdGUocGVyY2VudCA9IG5vZGVfc3VtL3NvdXJjZV90b3RhbCoxMDApICU+JSANCiAgbGVmdF9qb2luKG5vZGVfbWFwcGluZ19lKSAlPiUgDQogIGFycmFuZ2Uobm9kZSkgJT4lIA0KICBtdXRhdGUoeCA9IC4wMSkgJT4lIA0KICBtdXRhdGUoY3N1bV9ub3JtID0gc291cmNlX3RvdGFsKQ0Kc291cmNlX3BlcmNlbnQkY3N1bSA8LSBjdW1zdW0oc291cmNlX3BlcmNlbnQkbm9kZV9zdW0pDQpzb3VyY2VfcGVyY2VudCRzdGFydCA8LSBsYWcoc291cmNlX3BlcmNlbnQkY3N1bSkNCg0KIyB0YXJnZXQNCnRhcmdldF9zdW0gPC0gbGlua3NfZGF0YSAlPiUgDQogIHNlbGVjdChUYXJnZXRfbGFiZWwsIHZhbHVlKSAlPiUgDQogIGxlZnRfam9pbihub2RlX21hcHBpbmdfZSAlPiUgc2VsZWN0KGxhYmVsLCBzdGFnZSksIGJ5ID0gYygnVGFyZ2V0X2xhYmVsJyA9ICdsYWJlbCcpKSAlPiUgDQogIHJlbmFtZShsYWJlbD1UYXJnZXRfbGFiZWwpICU+JSANCiAgZmlsdGVyKHN0YWdlID09ICd0YXJnZXQnKSAlPiUgDQogIGdyb3VwX2J5KGxhYmVsLCBzdGFnZSkgJT4lIA0KICBzdW1tYXJpemUobm9kZV9zdW0gPSBzdW0odmFsdWUpKQ0KDQp0YXJnZXRfdG90YWwgPC0gdGFyZ2V0X3N1bSAlPiUgDQogIHB1bGwobm9kZV9zdW0pICU+JSBzdW0NCg0KdGFyZ2V0X3BlcmNlbnQgPC0gdGFyZ2V0X3N1bSAlPiUgDQogIG11dGF0ZShwZXJjZW50ID0gbm9kZV9zdW0vdGFyZ2V0X3RvdGFsKjEwMCkgJT4lIA0KICBsZWZ0X2pvaW4obm9kZV9tYXBwaW5nX2UpICU+JSANCiAgYXJyYW5nZShub2RlKSAlPiUgDQogIG11dGF0ZSh4ID0gLjk1KSAlPiUgDQogIG11dGF0ZShjc3VtX25vcm0gPSB0YXJnZXRfdG90YWwpDQp0YXJnZXRfcGVyY2VudCRjc3VtIDwtIGN1bXN1bSh0YXJnZXRfcGVyY2VudCRub2RlX3N1bSkNCnRhcmdldF9wZXJjZW50JHN0YXJ0IDwtIGxhZyh0YXJnZXRfcGVyY2VudCRjc3VtKQ0KDQojIEludGVybWVkaWF0ZSBDYXJyaWVycyBGbG93cyBpbg0KaW50ZXJtZWRpYXRlX25vZGVzIDwtIG5vZGVfbWFwcGluZ19lICU+JSBmaWx0ZXIoc3RhZ2UgPT0gJ21pZCcpICU+JSBwdWxsKGxhYmVsKQ0KIGludGVybWVkaWF0ZV9mbG93c19pbl90b3RhbCA8LSBsaW5rc19kYXRhICU+JQ0KICAgZmlsdGVyKFRhcmdldF9sYWJlbCAlaW4lIGludGVybWVkaWF0ZV9ub2RlcykgJT4lIA0KICAgZ3JvdXBfYnkoVGFyZ2V0X2xhYmVsKSAlPiUgDQogICBzdW1tYXJpemUobm9kZV9zdW0gPSBzdW0odmFsdWUpKQ0KIA0KaW50ZXJtZWRpYXRlX3BlcmNlbnQgPC0gaW50ZXJtZWRpYXRlX2Zsb3dzX2luX3RvdGFsICU+JSANCiByZW5hbWUobGFiZWwgPSBUYXJnZXRfbGFiZWwpICU+JSANCiBtdXRhdGUoc3RhZ2UgPSAnbWlkJykgJT4lIA0KIG11dGF0ZShwZXJjZW50ID1ub2RlX3N1bS9zb3VyY2VfdG90YWwqMTAwKSAlPiUgDQogbGVmdF9qb2luKG5vZGVfbWFwcGluZ19lKQ0KDQppbnRlcm1lZGlhdGVfdG90YWwgPC0gaW50ZXJtZWRpYXRlX3BlcmNlbnQgJT4lIHB1bGwobm9kZV9zdW0pICU+JSBzdW0NCg0KaW50ZXJtZWRpYXRlX2Zsb3dzX291dF90b3RhbCA8LSBsaW5rc19kYXRhICU+JQ0KIGZpbHRlcihTb3VyY2VfbGFiZWwgJWluJSBpbnRlcm1lZGlhdGVfbm9kZXMpICU+JSANCiBncm91cF9ieShTb3VyY2VfbGFiZWwpICU+JSANCiBzdW1tYXJpemUodmFsdWUgPSBzdW0odmFsdWUpKQ0KICANCmBgYA0KDQoNCmBgYHtyIGZpZy53aWR0aD04LCB3YXJuaW5nPUZBTFNFfQ0KIyBwcm9jZXNzIG5vZGUgbG9jYXRpb25zIA0KDQojIGZpbmFsIG5vZGUgaW5mbw0Kbm9kZXNfZGF0YSA8LSBiaW5kX3Jvd3Moc291cmNlX3BlcmNlbnQsIGludGVybWVkaWF0ZV9wZXJjZW50LCB0YXJnZXRfcGVyY2VudCkgJT4lDQogIGFycmFuZ2Uobm9kZSkgJT4lDQogIHJlcGxhY2VfbmEobGlzdChzdGFydCA9IDApKSAlPiUgDQogIG11dGF0ZShtaWRfcG9pbnQgPSAoc3RhcnQrY3N1bSkvMikgJT4lIA0KICBtdXRhdGUoeSA9IG1pZF9wb2ludC9jc3VtX25vcm0pICU+JSANCiAgbXV0YXRlKHkgPSBpZmVsc2UobGFiZWwgPT0gJ0dhcycsIDAuNSwNCiAgICAgICAgICAgICAgICAgICAgaWZlbHNlKGxhYmVsID09ICdMaXF1aWQgRnVlbHMnLCAwLjIsDQogICAgICAgICAgICAgICAgICAgIGlmZWxzZShsYWJlbCA9PSAnRWxlY3RyaWNpdHknLCAwLjYsDQogICAgICAgICAgICAgICAgICAgIGlmZWxzZShsYWJlbCA9PSAnSHlkcm9nZW4nLDAuOSx5KSkpKSkgJT4lIA0KICBtdXRhdGUoeCA9IGlmZWxzZShsYWJlbCA9PSAnR2FzJywgMC4yNSwNCiAgICAgICAgICAgICAgICAgICAgaWZlbHNlKGxhYmVsID09ICdMaXF1aWQgRnVlbHMnLCAwLjQsDQogICAgICAgICAgICAgICAgICAgICAgICAgICBpZmVsc2UobGFiZWwgPT0gJ0VsZWN0cmljaXR5JywgMC42LA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGlmZWxzZShsYWJlbCA9PSAnSHlkcm9nZW4nLCAwLjcseCkpKSkpICU+JQ0KICBtdXRhdGUobm9kZV9sYWJlbCA9IGlmZWxzZShpcy5uYShub2RlX3N1bSksIGxhYmVsLCANCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBwYXN0ZTAobGFiZWwsICcgJyxyb3VuZChub2RlX3N1bSwgZGlnaXRzID0gMSkgLCBnY2FtX2RhdGFfdW5pdCwgDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICcgJywgcm91bmQocGVyY2VudCwgZGlnaXRzID0gMSksJyUnKSkpIA0KICANCg0KIyBDaGVjayB0aGF0IFNvdXJjZSBhbmQgVGFyZ2V0cyBpbiBMaW5rcyBhcmUgaW4gdGhlIG5vZGUgbWFwcGluZw0KDQppZiggYW55KGlzLm5hKGxpbmtzX2RhdGEkU291cmNlX25vZGUpKSApIHN0b3AoIkNoZWNrIFNvdXJjZSBudW1iZXIgbWFwcGluZyAtIE5BJ3MiKQ0KaWYoIGFueShpcy5uYShsaW5rc19kYXRhJFRhcmdldF9ub2RlKSkgKSBzdG9wKCJDaGVjayBUYXJnZXQgbnVtYmVyIG1hcHBpbmcgLSBOQSdzIikNCiAgDQpkYXRhdGFibGUobm9kZXNfZGF0YSwgZmlsdGVyID0gJ3RvcCcsIHJvd25hbWVzID0gRkFMU0UsIG9wdGlvbnMgPSBsaXN0KHBhZ2VMZW5ndGggPSAyMCwgc2Nyb2xsWCA9IFRSVUUpKQ0KYGBgDQoNCg0KYGBge3IgZmlnLndpZHRoPTgsIHdhcm5pbmc9RkFMU0V9DQojIHNhdmUgZmlsZXMgZm9yIEtlbmRhbGwNCndyaXRlX2NzdihTb3VyY2VfVGFyZ2V0X2FsbF9sb3NzZXMsIHBhc3RlMCgiLi4vIiwgZGF0YV9kaXIsICdhbGxlbmVyZ3lfc291cmNlX3RhcmdldC5jc3YnKSkNCndyaXRlX2Nzdihub2Rlc19kYXRhLCBwYXN0ZTAoIi4uLyIsIGRhdGFfZGlyLCAnYWxsZW5lcmd5X25vZGVzX2RhdGEuY3N2JykpDQp3cml0ZV9jc3YobGlua3NfZGF0YSwgcGFzdGUwKCIuLi8iLCBkYXRhX2RpciwgJ2FsbGVuZXJneV9saW5rc19kYXRhLmNzdicpKQ0KDQpgYGANCg0KDQpgYGB7ciBmaWcud2lkdGg9OCwgd2FybmluZz1GQUxTRX0NCiMgcGxvdCBzYW5rZXkNCnNhbmtleV9maWd1cmUgPC0gcGxvdF9seSggDQogICAgICB0eXBlID0gInNhbmtleSIsDQogICAgICAjIGFycmFuZ2VtZW50ID0gInNuYXAiLA0KICAgICAgZG9tYWluID0gbGlzdCh4ID0gIGMoMCwxKSx5ID0gIGMoMCwxKSksDQogICAgICBvcmllbnRhdGlvbiA9ICJoIiwNCiAgICAgIHZhbHVlZm9ybWF0ID0gIi4wZiIsDQogICAgICB2YWx1ZXN1ZmZpeCA9IGdjYW1fZGF0YV91bml0LA0KDQojIE5vZGVzICANCiAgICAgIG5vZGUgPSBsaXN0KCBsYWJlbCA9IG5vZGVzX2RhdGEgJT4lIHB1bGwobm9kZV9sYWJlbCksDQogICAgICAgICAgICAgICAgICAgY29sb3IgPSBub2Rlc19kYXRhICU+JSBwdWxsKGhleCksDQogICAgICAgICAgICAgICAgICAgeCA9IG5vZGVzX2RhdGEgJT4lIHB1bGwoeCksDQogICAgICAgICAgICAgICAgICAgeSA9IG5vZGVzX2RhdGEgJT4lIHB1bGwoeSksDQogICAgICAgICAgICAgICAgICAgcGFkID0gMywNCiAgICAgICAgICAgICAgICAgICB0aGlja25lc3MgPSAxNSwNCiAgICAgICAgICAgICAgICAgICBsaW5lID0gbGlzdChjb2xvciA9ICJibGFjayIsd2lkdGggPSAwLjUpKSwNCiAgDQojIExpbmtzDQogICAgICBsaW5rID0gbGlzdChzb3VyY2UgPSBsaW5rc19kYXRhJFNvdXJjZV9ub2RlLA0KICAgICAgICAgICAgICAgICAgdGFyZ2V0ID0gbGlua3NfZGF0YSRUYXJnZXRfbm9kZSwNCiAgICAgICAgICAgICAgICAgIHZhbHVlID0gIGxpbmtzX2RhdGEkdmFsdWUsDQogICAgICAgICAgICAgICAgICBjb2xvciA9ICBsaW5rc19kYXRhJHJnYmEpDQopIA0KDQojIGFkZCBGb3JtYXR0aW5nDQpwbG90X3RpdGxlIDwtIHBhc3RlMCgnRW5lcmd5IC0gJywgcGxvdF9zY2VuYXJpb19uYW1lLCAnIC0gJyxzZWxlY3RfeWVhcikNCnNhbmtleV9maWd1cmUgPC0gc2Fua2V5X2ZpZ3VyZSAlPiUgbGF5b3V0KA0KICB0aXRsZSA9IHBsb3RfdGl0bGUsDQogIGZvbnQgPSBsaXN0KHNpemUgPSAxMSksDQogIHhheGlzID0gbGlzdChzaG93Z3JpZCA9IEYsIHplcm9saW5lID0gRiksDQogIHlheGlzID0gbGlzdChzaG93Z3JpZCA9IEYsIHplcm9saW5lID0gRikpDQoNCnNhbmtleV9maWd1cmUNCg0KDQpgYGANCg0KDQoNCmBgYHtyIGZpZy53aWR0aD04LCB3YXJuaW5nPUZBTFNFfQ0KDQoNCmBgYA0KDQoNCg0KDQoNCg0K